1 module dataframe.hdf5util; 2 import hdf5.hdf5; 3 import std.conv; 4 import std.csv; 5 import std.datetime; 6 import std.exception; 7 import std.range:array, stride,only; 8 import std.stdio; 9 import std.variant; 10 import std.string:isNumeric; 11 alias KalVariant=Algebraic!(string,int,long, DateTime, float,double); 12 import std.typecons:tuple,Tuple; 13 14 extern(C) herr_t H5_init_library(); 15 16 enum DumpMode 17 { 18 unlink, 19 truncate, 20 append, 21 } 22 23 hsize_t getUserBlockSize(string filename) 24 { 25 hsize_t usize; 26 auto testval = H5F.is_hdf5(filename); 27 enforce(testval>0, new Exception("Input HDF5 file is not HDF: "~filename)); 28 auto ifile = H5F.open (filename, H5F_ACC_RDONLY, H5P_DEFAULT); 29 enforce(ifile>=0, new Exception("Cannot open input HDF5 file: "~filename)); 30 auto plist = H5F.get_create_plist (ifile); 31 enforce(plist>=0, new Exception("Cannot get file creation plist for file "~filename)); 32 H5P.get_userblock (plist, &usize); 33 H5P.close (plist); 34 H5F.close (ifile); 35 return usize; 36 } 37 38 void setUserBlock(string filename, ubyte[] buf) 39 { 40 auto usize=getUserBlockSize(filename); 41 if (usize<buf.length) 42 throw new Exception("Attempted to set user block for file: "~ filename~ " but user block is only "~ 43 to!string(usize) ~ " bytes long and buffer is "~to!string(buf.length)~" bytes long"); 44 auto f=File(filename,"wb+"); 45 f.rewind(); 46 f.rawWrite(buf); 47 f.flush(); 48 f.close(); 49 } 50 51 ubyte[] getUserBlock(string filename) 52 { 53 ubyte[] buf; 54 auto usize=getUserBlockSize(filename); 55 buf.length=cast(size_t) usize; 56 auto f=File(filename,"rb+"); 57 f.rewind(); 58 auto numbytes=f.rawRead(buf); 59 buf.length=numbytes.length; 60 f.close(); 61 return buf; 62 } 63 64 hid_t friendlyH5Create(string filename, hsize_t userBlockSize, bool truncateNotThrow ) 65 { 66 import std.file:exists; 67 H5open(); 68 H5_init_library(); 69 if ((!truncateNotThrow)&&exists(filename)) 70 throw new Exception("friendlyH5Create: attempt to create file that already exists: "~filename); 71 userBlockSize=computeUserBlockSize(userBlockSize); 72 writefln("%s userblock size",userBlockSize); 73 auto plist = H5Pcreate(H5P_FILE_CREATE); //H5P_FILE_CREATE); 74 //writefln("%s",H5P.get_class_name(plist)); 75 //stdout.flush; 76 //auto plist=H5P_DEFAULT; 77 H5P.set_userblock(plist, userBlockSize) ; 78 auto file_id = H5F.create(filename, H5F_ACC_TRUNC, plist, H5P_DEFAULT); 79 //auto file_id = H5F.create(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); 80 H5P.close(plist); 81 return file_id; 82 } 83 84 hsize_t computeUserBlockSize (hsize_t ublock_size) 85 { 86 hsize_t where = 512; 87 if (ublock_size == 0) 88 return 0; 89 while (where < ublock_size) 90 where *= 2; 91 return (where); 92 } 93 94 95 96 string[] hdf5Contents(string filename) 97 { 98 import std.exception:enforce; 99 import std.file:exists; 100 enforce(filename.exists,new Exception("hdf5Contents - file: "~filename~" does not exist!")); 101 auto file=H5F.open(filename,H5F_ACC_RDWR, H5P_DEFAULT); 102 auto files=cast(string[])objectList(file); 103 H5F.close(file); 104 return files; 105 } 106 107 /+ 108 bool isEquityMain(hid_t group, string ticker, string filename) 109 { 110 import hdf5.wrap; 111 import hdf5.bindings.enums; 112 import hdf5.bindings.api; 113 if (!filename.toLower.canFind("equitymain")) 114 return false; 115 auto dataset = H5D.open2(group, ticker, H5P_DEFAULT); 116 auto dataspace = H5D.get_space(dataset); /* dataspace handle */ 117 auto rank = H5S.get_simple_extent_ndims(dataspace); 118 H5D.close(dataset); 119 if (rank>=3) 120 throw new Exception("isEquityMain: unknown data format for "~filename~"; rank="~rank.to!string); 121 return (rank==2); 122 } 123 +/